	package navigation.results.cluster;

	import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileFilter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import navigation.results.filter.ContainsInAttribute;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.lexer.Lexer;
import org.htmlparser.lexer.Page;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

import wish.stages.ParsingStage;

	public class NextBasedCluster {

		private static final String HREF_ATTRIBUTE = "href";
				
		public class NextBasedClusterRunnable implements Runnable{

			private File input;
			private File output;
			private NodeFilter nf;
			

			public NextBasedClusterRunnable(File file, File outp, NodeFilter nf) {
				
				this.input = file;
				this.output = outp;
				this.nf = nf;
			}

			@Override
			public void run() {
							
				try {
					
					String htmlContent = ParsingStage.readFile(input);
					
					Parser parser = new Parser(new Lexer(new Page(htmlContent, "UTF-8")));
					
					NodeList n = parser.parse(nf);
					
					BufferedWriter bw = new BufferedWriter(new FileWriter(output));
					
					boolean theClusterFound = false;
					
					int numberOfTheCluster = 1;
					
					for (int i = 0; i < n.size(); i++) {
						
						bw.write("<div");
						
						Node node = n.elementAt(i);
						
						String text = ParsingStage.recoverText(node).toLowerCase();
						
						if (text.contains("next")){
							bw.write(" id=\""+ numberOfTheCluster+ "\"><h1> THE CLUSTER: </h1>\n");
							theClusterFound = true;
							numberOfTheCluster++;
						}else{
							bw.write(">\n<h2> Cluster: </h2>\n");
						}
						
						bw.write("<br>");
						
						bw.write(node.toHtml());
						
					}
					
					if (!theClusterFound){
						
						System.err.println(output.getAbsolutePath());
						
					}
					
					bw.close();
				
				} catch (ParserException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
				
			}

			
		}

		private static Set<String> toProcess;
		
		/**
		 * @param args
		 * @throws IOException 
		 */
		public static void main(String[] args) throws IOException {
			
			toProcess = loadToProcess("/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/NavigationReportComplexText/bad");
			
			String navFolder = "/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/Navigation/";
			
			File outp = new File("/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/NavigationClusterComplexTextNext/");
			
			File tfidfFolder = new File("/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/tfidf/");
			
			File fold = new File(navFolder);
			
			File[] navfolders = fold.listFiles(new FileFilter() {
				
				@Override
				public boolean accept(File pathname) {
					return toProcess.contains(pathname.getName());
				}
			});
			
			//Process All
			
//			navfolders = fold.listFiles();
			
			for (int i = 0; i < navfolders.length; i++) {
				
				System.out.println("DATABASE: " + i);
				
				File folder = navfolders[i];
				
				String name = folder.getName();
				
				List<String> queries = loadQueries(new File(tfidfFolder,name));
				
				File[] files = folder.listFiles();
				
				File dir = new File(outp,name);
				
				if (dir.exists())
					continue;
				
				dir.mkdir();
				
				System.setErr(new PrintStream(new File(dir, "notFound.txt")));
				
				for (int j = 0; j < files.length; j++) {
					
					System.gc();
					
					int index = Integer.valueOf(files[j].getName().replace(".html", ""));
					
					NodeFilter nf = new AndFilter(new HasAttributeFilter(HREF_ATTRIBUTE), new ContainsInAttribute(HREF_ATTRIBUTE,"="+queries.get(index)));
					
					System.out.println("FILE: " + j);
					
					new NextBasedCluster().execute(files[j],new File(dir,files[j].getName()),nf);
					
				}
				
			}

		}

		private static Set<String> loadToProcess(String fileName) throws IOException {
			
			Set<String> set = new HashSet<String>();
			
			BufferedReader br = new BufferedReader(new FileReader(fileName));
			
			String line;
			
			while((line=br.readLine())!=null){
				
				set.add(line);
				
			}
			
			br.close();
			
			return set;
			
		}

		private void execute(File file, File outp, NodeFilter nf) {
			
			Thread t = new Thread(new NextBasedClusterRunnable(file,outp,nf));
			
			t.start();
			
			try {
				t.join();
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			
		}
	
		private static List<String> loadQueries(File file) throws IOException {
			
			List<String> ret = new ArrayList<String>();
			
				
			BufferedReader br = new BufferedReader(new FileReader(file));
			
			String line;
			
			while ((line=br.readLine())!=null){
				
				String[] l = line.split(" ");
				
				ret.add(l[0]);
				
			}
			
			br.close();
			
			return ret;
			
		}
		
}
